########################################################################
#  Pyfm - personal finance data-base
#  Copyright (C) 2007,8 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
""" Commannd line based application for handling all data
Commands make sense according the following order:

help,? or ENTER - type help on commands.
? <Command> - type help on command.
help <Command> - type help on command.

lf - load flat files to DB in memory and build objects
lx - enter flat line manually directly to DB and build objects
c - control debug prints
s - save DB (lines and objects) in the pkl file.
l - load DB from the pkl file.

i - print statisitical information on the DB content.
p <Filter> - print all lines that match filter. 
    A filter matches a line if for *all* tags in the filter there is an overlap in any of its
    values with values the same tag received in the line. For example:
    p Line!PRINT HIDX=23
    Will print all lines that contain the tag=value pair HIDX=23.
    
    Its possible to define in the filter multiple values by placing
    several tag=value pairs in the filter. Its enough, however, that only one of these values will appear
    in the matched line. For example:
    p Line!PRINT HIDX=23 HIDX=340
    Will print all lines that contain the tag=value pair HIDX=23 or HIDX=340 (or both)
    
    Its possible surround a value with {...}. In this case a the value assigned to the tag
    will be looked for in all the lines that match the filter defined inside the {...}
        For example:
    p Line!PRINT Holding={HIDX=23}
    Will print all lines in which the Holding appeared that also appeared in at least one of the lines for which
    the pair HIDX=23 also appeared. Effectivly we are using HIDX=23 to define the Holding we are interested in
    and then we print all the lines in which that Holding appeared.

        It's possible to add to Filter tag-value pairs of the following syntax:
    !<tag>=SORT
    and in that case the objects will be sorted according to their <tag> value before printing.
    MAX and MIN will accordingly keep the maximal/minimal value of the tag.

Session example:
help # print help
? lf # print help on the lf command
lf # loaded flatten files
p DTASOF!SORT DTASOF!PRINT UNITPRICE!PRINT Asset={AIDX=127}  # Print price history sorted by date of Asset with AIDX=127
s # save
q # quit program

Another session:
l # Load previously load database
p AIDX!PRINT ASSETCLASS!PRINT Asset!UNIQUE Asset={Holding={ACCTGRP=N Holding={HoldingSource={Statement={UNITS>0 DTASOF>20-Dec-06}}}}}
   # Print AIDX and ASSETCLASS of Asset currently (as of 20/12/06) in use in bank N
p Transaction!PRINT FILE=_MERGED_ DTSETTLE!SORT DTSETTLE>=1-Nov-06 Holding={ACCTGRP=A}
   # Print merged transactions of bank A starting from 1/11/06
p Transaction!PRINT TRNTYPE=DIV -PPRINT FILE=_MERGED_ Holding={ACCTGRP=N} DTSETTLE<1-Jan-2007 DTSETTLE>=1-Jan-06 DTSETTLE!SORT
   # Print the content of dividend transaction in bank N in 2006
p TRNTYPE!GET # List all TRNTYPE that are in use in the database

Another session:
lf FILE=xxx.flt # Load a specifice flat file
p SECID!GET SECID!SORT INTEREST!SUM TRNTYPE=div COUNTRY=USA
  # Sorted list all assets from USA and for each sum their dividend
"""
import logging
import os, time
from cmd import Cmd
from flat import flat2objdb, parse, stagvalue,print_line
from obj import db,Obj
from objtools import print_lines,print_objs,print_lines_content,save_lines
from lib import dirs
from lib import conf
from lib.quoteinterface import getQuote
from dt import DT
import webbrowser

class MyCmd(Cmd):
    prompt='>'
    intro="""pyfm, Copyright (C) 2007,8 Ehud Ben-Reuven
pyfm comes with ABSOLUTELY NO WARRANTY; for details
see GPL 2.0.  This is free software, and you are welcome
to redistribute it under certain conditions; See GPL 2.0 
for details."""
       
    def loadfiles(self,line):
        """Load flat files, look for FILE values in line that can be either file names or directories
        by default load all files from $PYFM_DATA\flt and its subdirecories
        """
        for fname in dirs.globLine(line,dirs.flt()):
            if os.path.isdir(fname):
                # Load all flat files found in the directory or in any of its subdirectories
                # or fname itself if it is a file.
                def visit(fnlist,root,names):
                    skip_list=[i for i in range(len(names)) if names[i].startswith('.')]
                    skip_list.reverse()
                    for i in skip_list:
                        del names[i]
                    for name in names:
                        r,ext=os.path.splitext(name)
                        if ext!='.txt': continue
                        fn=os.path.join(root,name)
                        if not os.path.isfile(fn): continue
                        fnlist.append(fn)
                fnlist=[]
                os.path.walk(fname,visit,fnlist)
                fnlist.sort()
                for fn in fnlist:
                    print 'Loading',fn
                    flat2objdb(fn,db)
            else:
                print 'Loading',fname
                flat2objdb(fname,db)
                
    
    def onecmd(self, line):
        """Override of Cmd.onecmd
        """
        cmd, arg, line = self.parseline(line)
        if not line: return
        if line.startswith('#'): return # Skip comment lines
        if cmd is None:
            return self.default(line)
        self.lastcmd = line
        if cmd == '':
            return self.default(line)
        else:
            try:
                func = getattr(self, 'do_' + cmd)
            except AttributeError:
                return self.default(line)
        if cmd=='help':
            return func(arg)
        fltline=parse(arg)
        if conf.get('PASS','TRUE')[0]!='T' or self.batch_mode:
            return func(fltline)
        else:
            try:
                return func(fltline)
            except Exception,e:
                print
                print `e`
    #def emptyline(self,line={}): self.do_help(line)
    def default(self,line={}): self.do_help(line)
    def do_EOF(self,line={}):
        """Quit"""
        return True
    def do_q(self,line={}):
        """Quit"""
        return True
    def do_i(self,line={}):
        """Statistics on objects in DB"""
        print "Line=",len(db.lines)
        for cls in db.clsdb:
            print "%d %s"%(len(db.getallobjects(cls)),cls.__name__)
    def do_l(self,line={}):
        """Load DB from pkl file"""
        fnames=dirs.globLine(line, dirs.root('data.pkl'))
        if len(fnames)>1: raise Exception('Cant handle more than one file name')
        db.load(fnames[0])
    def do_s(self,line={}):
        """Save DB to pkl file"""
        fnames=line.get('FILE',[dirs.root('data.pkl')])
        if len(fnames)>1: raise Exception('Cant handle more than one file name')
        db.dump(fnames[0])
    # TODO add d command that looks for _LINE_INDEX_ or Line tags,  use them to find the lines in the data-base and delete them.
    def do_lx(self,line={}):
        # TODO instead of calling this lx command, call it u command
        # TODO replace all the values that are dictonaries with a query to the data base using the old value as a filter and the tag as the thing we want to get.
        # TODO allow user to specify '_LINE_INDEX_' tag to update existing line(s)
        # TODO allow user to specify 'Line' tag to update existing line(s)
        #
        """Manual load a single line to DB."""
        line['_INTERACTIVE']=[True]
        db.line_update(line)
    def do_lf(self,line={}):
        """load flat files to memory and generate objects in the process.
All flat files located in $PYFM_DATA/flt subdirectories are loaded
OR
use flt FILE=XXX.txt to load the single file XXX.txt
"""
        self.loadfiles(line)
    def do_po(self,line):
        """po <class>=<filter> [-PFULL | -PPRINT | <tag>!PRINT*] [<tag>!SORT | <tag>!MAX | <tag>!MIN]
Use <class>!PRINT to get an exact filter for each object found.
"""
        print_objs(line)
    def do_p(self,line={}):
        """p <filter>
Query the DB with the filter and print the results.
"""
        print_lines_content(line)
    def do_pl(self,line={}):
        """pl [-NO] [-V] <filter>
Query the DB with the filter and print all flat lines that match the filter.
-NO - display only the original line content without computed objects
"""
        print_lines(line)
    def do_sl(self,line={}):
        """sl [-V] FILE=<output-file> Line={<filter>}
Query the DB with the filter and write as text lines all flat lines that match the filter.
"""
        save_lines(line)
    def do_c(self,line):
        """Control program run time configure:
c [LOGGING=DEBUG|INFO|WARNING|ERROR|CRITICAL] [DATA=<data-directory>] [ONLINE=TRUE|FALSE] [PASS=TRUE|FALSE]
"""
        for key in line:
            conf.manual_set(key, line[key][0])
    def do_quote(self,line):
        """Get on-line quote:
quote (TICKER=<ticker>|SECID=<security-identifier>) [DTASOF=<date>]"""
        for secid in line.get('SECID',[]):
            for dt in line.get('DTASOF',[DT('NOW')]):
                p,dtp=getQuote(dt,secid)
                print stagvalue('SECID',secid),stagvalue('UNITPRICE',p),stagvalue('DTASOF',dtp)
        for ticker in line.get('TICKER',[]):
            for dt in line.get('DTASOF',[DT('NOW')]):
                p,dtp=getQuote(dt,'TICKER='+ticker)
                print stagvalue('TICKER',ticker),stagvalue('UNITPRICE',p),stagvalue('DTASOF',dtp)
    def do_help(self, arg):
        if  'T' == conf.get('ONLINE')[0]:
            url="http://benreuven.com/udiwiki/index.php?title=Pyfm:CommandShell"
            if arg: url+='#'+arg
            webbrowser.open(url)
        else:
            Cmd.do_help(self,arg)
    batch_mode=0
    def do_x(self,line={}):
        """Execute commands from a script file"""
        fname=line['FILE'][0]
        fname=dirs.script([fname+'.txt'])
        print 'Executing',fname
        fin = open(fname,'rb') # The 'b' is needed because asset names can have unicode values
        self.batch_mode+=1 # Use counter to allow the x command to be recursive
        for l in fin:
            l=l.strip()
            print l
            if self.onecmd(l):
                fin.close()
                return True
        self.batch_mode-=1
        fin.close()
    def do_clc(self,line={}):
        """Clean the screen"""
        for i in range(120): print

    def preloop(self):
        """Welcome to pyfm. For an introduction visit:
http://benreuven.com/udi/pyfm

Command line options:
pyfm [-h] [-help]
pyfm [-c shell-command] [--execute=shell-command]
pyfm [-x script-file] [--script=script-file]

-h, -help: this help.
-c,--execute: execute the following single shell command
  before starting the shell session.
  You can enter multiple commands that will be executed
  in order.
  If a shell-command has arguments it should be  protected
  with quotes.
  If you don't want to enter an interactive command shell
  then just specificy "q" as the last command
  For more details on shell commands visit
benreuven.com/udi/pyfm/wiki/index.php?title=CommandShell
-x, --script: execute a script file located in the directory
  $PYFM_DATA/script.
  This is equivalent to "x FILE=script-file" shell command.
  
For example, all the following do the same:
   pyfm -x all.txt
   pyfm --script=all.txt
   pyfm -c"x FILEall.txt"
   pyfm --execute="x FILE=all.txt"
and with the default content of data/script/all.txt this will be
the same as:
  pyfm -c lf -c fa -c mt -c ms -c s -c q
"""
        def usage(doc=self.preloop.__doc__,intro=self.intro):
            print doc
            print intro
        try:
            import getopt
            opts, args = getopt.getopt(os.sys.argv[1:], "hc:x:", ["help", "execute=","script="])
        except getopt.GetoptError:
            # print help information and exit:
            usage()
            os.sys.exit(2)
        for o, a in opts:
            if o in ("-h", "--help"):
                usage()
                os.sys.exit(0)
            if o in ("-c", "--execute"):
                print '>'+a
                if self.onecmd(a): os.sys.exit(0)
            if o in ("-x", "--script"):
                print '>'+a
                if self.onecmd('x FILE="'+a+'"'): os.sys.exit(0)
if __name__=='__main__': MyCmd().cmdloop()
